# -*- coding: utf-8 -*-
import logging
import scrapy
import re
import json
from douban_movie.items import DoubanMovieReviewItem, DoubanMovieReviewDetailItem

# 电影id
MOVIE_ID = 26794435

class ReviewsSpider(scrapy.Spider):
    name = 'reviews'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/subject/%d/reviews?sort=hotest&start=4620' % MOVIE_ID]
    def parse(self, response):
        """
        爬取影评列表
        :param response:
        :return:
        """
        review_list = response.xpath('//div[contains(@class,"review-list")]/div')
        if review_list:
            for review in review_list:
                item = DoubanMovieReviewItem()
                # 影评id
                # 有多种写法，extract_first()返回第一个，未获取到值则可以在extract_first([默认值])设置默认值，extract()返回的是集合形式
                id = review.xpath('@data-cid').extract_first()
                # 头像
                avatar = review.xpath('div/header/a[@class="avator"]/img/@src').extract_first()
                # 昵称
                nick = review.xpath('div/header/a[@class="name"]/text()').extract_first().strip()
                # 评级
                level = review.xpath('div/header/span[contains(@class, "main-title-rating")]/@title').extract_first('未打分')
                # 时间
                create_time = review.xpath('div/header/span[@class="main-meta"]/text()').extract_first().strip()
                # 影评标题
                title = review.xpath('div/div/h2/a/text()').extract_first().strip()
                action = review.xpath('div/div/div[@class="action"]')
                # 顶
                up = action.xpath('a[contains(@class,"up")]/span/text()').extract_first('0').strip()
                # 踩
                down = action.xpath('a[contains(@class,"down")]/span/text()').extract_first('0').strip()
                # 回复数
                reply_str = action.xpath('a[contains(@class, "reply")]/text()').extract_first().strip()

                reply = int(re.sub(r'\D+', '', reply_str))
                item['id'] = id
                item['movie_id'] = MOVIE_ID
                item['avatar'] = avatar
                item['nick'] = nick
                item['level'] = level
                item['create_time'] = create_time
                item['title'] = title
                item['up'] = up
                item['down'] = down
                item['reply'] = reply
                yield item
                detail_url = 'https://movie.douban.com/j/review/%s/full' % id
                yield scrapy.Request(url=detail_url, callback=self.detail_parse)

        # 如果还有下一页则继续爬取
        next_params = response.selector.xpath('//div[@class="paginator"]/span[@class="next"]/a/@href').extract_first()
        if next_params:
            next_url = 'https://movie.douban.com/subject/%d/reviews?sort=hotest&start=%s' % (MOVIE_ID, next_params.split('=')[-1])
            yield scrapy.Request(url=next_url, callback=self.parse)

    def detail_parse(self, response):
        # # 获取请求头
        # response.request.headers.getlist(参数名)
        # # 获取响应头
        # response.headers.getlist(参数名)
        #
        # 5.
        # response响应对象
        # response.url：当前响应的url地址
        # response.request.url：当前响应对应的请求的url地址
        # response.headers：响应头
        # response.body：响应体，也就是html代码，默认是byte类型
        # response.requests.headers：当前响应的请求头

        # 获取请求头
        url = response.request.url
        review_id = url.split('/')[-2]
        result = json.loads(response.text, encoding='utf-8')
        content = result['html']
        item = DoubanMovieReviewDetailItem()
        item['review_id'] = review_id
        item['content'] = content
        yield item






